import requests
from bs4 import BeautifulSoup

url = "https://www.shanghairanking.cn/rankings/bcur/2021"
headers = {
    "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/90.0.4430.212 Safari/537.36 Edg/90.0.818.62"}
response = requests.get(url, headers=headers)
response.encoding = 'utf-8'
html = response.text
soup = BeautifulSoup(html, 'lxml')
td_address = soup.find('div', {'id': 'content-box'}).find('table', {'class': 'rk-table'}).find('tbody').find_all('tr')
school_rank = []
for i in td_address:
    a = i.find_all('td')
    school_info = {}
    for j in a:
        if a.index(j) == 1:
            school_name = j.find('a').text
            school_info['school_name'] = school_name
        elif a.index(j) == 2:
            province = ''.join(j.text.strip().split())
            school_info['province'] = province
        elif a.index(j) == 3:
            school_type = ''.join(j.text.strip().split())
            school_info['school_type'] = school_type
        elif a.index(j) == 4:
            school_points = ''.join(j.text.strip().split())
            school_info['school_points'] = school_points
    school_rank.append(school_info)
    print(school_rank)

